package ac.man.cs.afzal.tcm.terms_sentence_handler;


import ac.man.cs.afzal.tcm.nlp.File_Manager_Impl;
import ac.man.cs.afzal.tcm.nlp.String_Manipulator_Impl;
import ac.man.cs.afzal.tcm.sql_handler.SQL_Handler_Terms_Impl;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.BufferedReader;
import java.io.File;
import java.util.Properties;
import java.util.Vector;
/**
 *
 * @author Hammad Afzal
 * @version 1.0
 * Modified on: 18th July, 2013 // Removed "Values" from Sentence Table
 */
public class Annotated_Sentence_Generator
{

    private String MODE = "MODE";
    private int MAX_SENTENCE_LENGTH = 1000;

    private String database_url = "database_url";
    private String username = "username";
    private String password = "password";

    private String ANNOTATED_DOCUMENTS_CORPUS = "ANNOTATED_DOCUMENTS_CORPUS";
    private String ANNOTATED_DOCUMENTS_TRAINING = "ANNOTATED_DOCUMENTS_TRAINING";

    private String ANNOTATED_SENTENCES_CORPUS = "ANNOTATED_SENTENCES_CORPUS";
    private String ANNOTATED_SENTENCES_TRAINING = "ANNOTATED_SENTENCES_TRAINING";


    private SQL_Handler_Terms_Impl sql_handler = new SQL_Handler_Terms_Impl();
    private File_Manager_Impl fileManager = new File_Manager_Impl();
    private String_Manipulator_Impl strMan = new String_Manipulator_Impl();

    public Annotated_Sentence_Generator()
    {
        Properties props = new Properties();
        try
        {
            props.load(new FileInputStream("Term_Classification.properties"));
            database_url = props.getProperty(database_url);
            username = props.getProperty(username);
            password = props.getProperty(password);

        }
        catch (Exception ex)
        {
            System.out.println("Exception while loading propoerties : " + ex.getCause());
        }

        MODE = props.getProperty(MODE);

        ANNOTATED_SENTENCES_CORPUS = props.getProperty(ANNOTATED_SENTENCES_CORPUS);
        ANNOTATED_SENTENCES_TRAINING = props.getProperty(ANNOTATED_SENTENCES_TRAINING);

        ANNOTATED_DOCUMENTS_CORPUS = props.getProperty(ANNOTATED_DOCUMENTS_CORPUS);
        ANNOTATED_DOCUMENTS_TRAINING = props.getProperty(ANNOTATED_DOCUMENTS_TRAINING);
    }


    /* It first reads the annotated document file, that has sentences and terms boundaries 
     * marked. It then reads the annotated sentences (with occurrence of terms are annotated),
     * and loads the values into Documents, Sentences and Occurrences tables of database.
     */
    public void countering_lookup() throws Exception
    {
        String[] tables =
        {
            "Occurrences", "Sentences", "Documents"
        };

        sql_handler.initialize_parameters(database_url, username, password);
        sql_handler.connect_db();

        BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
        String input;
        try
        {
            System.out.println(
                    "*****************************************************");
            System.out.println("You have opted to perform Sentence + Term Occurrence Extraction\n");
            System.out.print(
                    "WARNING: Your previous data in Documents,Sentences and Occurences \n " +
                    " table will be erased \n" +
                    " Enter 'y' to continue or 'n' to exit : ");
            input = in.readLine();
            input = input.trim();
            if ((input.equals("Y")) || (input.equals("y")))
            {
//                sql_handler.delete_all(tables);
//                sql_handler.reset_auto_inc(tables);
            }
        }
        catch (Exception ex)
        {
            System.out.println("Exception while taking input from user : " + ex.getMessage());
        }
        
        String input_dir = (MODE.equals("Training"))? ANNOTATED_DOCUMENTS_TRAINING: ANNOTATED_DOCUMENTS_CORPUS;

        
        File file_dir = new File(input_dir);
        String[] list_files = file_dir.list();
        int no_of_files = list_files.length;

        if (no_of_files == 0)
        {
            System.out.println("Sorry! There are no input files to be processed");
            System.exit(0);
        }

        int document_id = 0;
        int sentence_id = 0;

        String file_contents = "";

        for (String file_in_list: list_files)
        {
            int sentence_count = 0;
            int index = 0;

            String sentence_before_removing_lookup_tags = "";
            Vector terms_in_a_document = new Vector();
            File dir_per_term;

            String document_name = file_in_list.substring(0, file_in_list.length() - 6);
            String[] doc_table = {"Name", "Type"};
            String[] doc_table_value = {document_name,MODE };

            sql_handler.insert_table("Documents", doc_table, doc_table_value);
            document_id = sql_handler.get_document_id_per_doc_value(document_name);


            /************* Making Directory per Document for Output **************************/
            dir_per_term = (MODE.equals("Training"))? new File(ANNOTATED_SENTENCES_TRAINING + "//" + document_name):
                                                      new File(ANNOTATED_SENTENCES_CORPUS + "//" + document_name);

            dir_per_term.mkdir();
            /************* End Making Directory per Document for Output **************************/

            file_contents = fileManager.fileReader(input_dir + "//" + file_in_list).toString();

            while ((index = file_contents.indexOf("<Sentence>", index)) != -1)
            {
                int temp = index + 10;
                index = file_contents.indexOf("</Sentence>", temp);
                sentence_before_removing_lookup_tags = file_contents.substring(temp, index);

                if ((sentence_before_removing_lookup_tags.contains("<Candidte_Term>")) &&
                        (sentence_before_removing_lookup_tags.length() < 1000) && 
                        !(sentence_before_removing_lookup_tags.contains("MathType")))
                {
                    int index1 = 0;
                    int index2 = 0;
                    int index3 = 0;
                    String term = "";
                    Vector<String> terms_in_a_sentence = new Vector();

                    while ((index1 = sentence_before_removing_lookup_tags.indexOf("<Candidte_Term>", index1)) != -1)
                    {
                            index1 = index1 + 15;
                            index2 = sentence_before_removing_lookup_tags.indexOf("</Candidte_Term>", index1);
                            index3 = sentence_before_removing_lookup_tags.indexOf("<Candidte_Term>", index1);
                            term = sentence_before_removing_lookup_tags.substring(index1, index2);

                            if (index3 < index2)
                            {
                                String output = strMan.pattern_replacer(term, "<Candidte_Term>", "");
                                term = strMan.pattern_replacer(output, "<Candidte_Term>", "");
                            }
                            // 
                            // TODO: Check why if term contains "/", "\" or "}", the program stucks.
                            //
                            if ((!(term.contains("/"))) && (!(term.contains("{"))) &&(!(term.contains("\\"))) &&
                                    (!(term.contains("+"))) && (!(term.contains("*"))))
                            {
                                terms_in_a_sentence.add(term);
                            }
                    }

                    String output = strMan.pattern_replacer(sentence_before_removing_lookup_tags, "<Candidte_Term>", "");
                    String sentence_after_removing_lookup_tags = strMan.pattern_replacer(output, "</Candidte_Term>", "");
                    sentence_after_removing_lookup_tags = sentence_after_removing_lookup_tags.replaceAll("'", "");

                    if (sentence_after_removing_lookup_tags.length() > MAX_SENTENCE_LENGTH)
                    {
                        System.out.println("Sentence too long, Doc ID = " + document_id + "Sentence : " + sentence_after_removing_lookup_tags);
                        sentence_after_removing_lookup_tags = "SENTENCE TOO LONG";
                   }


                    for (int j = 0; j < terms_in_a_sentence.size(); j++)
                    {
                        String candidate_term = (String) terms_in_a_sentence.elementAt(j);
                        String sentence = sentence_after_removing_lookup_tags;
                        sentence = sentence.replaceAll(candidate_term, "MAIN_TERM");

                        /* Mark each term (non-main)
                        for (int k = 0; k < terms_in_a_sentence.size(); k++)
                        {
                            sentence = sentence.replace(terms_in_a_sentence.elementAt(k).toString(),
                                    "Term_" + terms_in_a_document.size() + "_" +
                                    k);
                        }
                        */
                        
                        terms_in_a_document.add(candidate_term);

                        String[] doc_table_1 = {"Doc_ID", "Local_Sen_ID"/*, "Value"*/};
                        String[] doc_table_value_1 = {Integer.toString(document_id), Integer.toString(sentence_count)/*, sentence_after_removing_lookup_tags*/};
                        sql_handler.insert_table("Sentences", doc_table_1, doc_table_value_1);

                        sentence_id = sql_handler.get_sentence_id(document_id, sentence_count);

                        String[] doc_table_2 = {"Value", "Doc_ID", "Sen_ID"};
                        String[] doc_table_value_2 = {candidate_term, Integer.toString(document_id), Integer.toString(sentence_id)};
                        sql_handler.insert_table("Occurrences", doc_table_2, doc_table_value_2);

                        if (MODE.equals("Training"))
                        {
                            if (sentence_count < 10)
                            {
                                fileManager.fileWriter(sentence,
                                        ANNOTATED_SENTENCES_TRAINING + "//" + document_name + "//00" +
                                        sentence_count + "__" + candidate_term + ".txt");
                            }
                            else if (sentence_count < 100)
                            {
                                fileManager.fileWriter(sentence,
                                        ANNOTATED_SENTENCES_TRAINING + "//" + document_name + "//0" +
                                        sentence_count + "__" + candidate_term + ".txt");
                            }
                            else
                            {
                                fileManager.fileWriter(sentence,
                                        ANNOTATED_SENTENCES_TRAINING + "//" + document_name + "//" +
                                        sentence_count + "__" + candidate_term + ".txt");
                            }
                        }
                        else
                        {
                            if (sentence_count < 10)
                            {
                                fileManager.fileWriter(sentence,
                                        ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//00" +
                                        sentence_count + "__" + candidate_term + ".txt");
                            }
                            else if (sentence_count < 100)
                            {
                                fileManager.fileWriter(sentence,
                                        ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//0" +
                                        sentence_count + "__" + candidate_term + ".txt");
                            }
                            else
                            {
                                fileManager.fileWriter(sentence,
                                        ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//" +
                                        sentence_count + "__" + candidate_term + ".txt");
                            }
                        }
                        sentence_count++;
                    }
                    terms_in_a_sentence.clear();
                }
            }
        }
        sql_handler.close_db();
    }

    public void countering_lookup_for_diease_gad() throws Exception
    {
        String[] tables =
        {
            "diseases_gad", "sentences_diseases_gad"
        };

        sql_handler.initialize_parameters(database_url, username, password);
        sql_handler.connect_db();
        sql_handler.reset_tables(tables);


        String input_dir = ANNOTATED_DOCUMENTS_CORPUS;


        File file_dir = new File(input_dir);
        String[] list_files = file_dir.list();
        int no_of_files = list_files.length;

        if (no_of_files == 0)
        {
            System.out.println("Sorry! There are no input files to be processed");
            System.exit(0);
        }

        int document_id = 0;
        int sentence_id = 0;

        String file_contents = "";

        for (String file_in_list: list_files)
        {
            int sentence_count = 0;
            int index = 0;

            String sentence_before_removing_lookup_tags = "";
            Vector terms_in_a_document = new Vector();
            File dir_per_term;

            String document_name = file_in_list.substring(0, file_in_list.length() - 4);
            document_id = sql_handler.get_document_id_per_doc_value(document_name);


            /************* Making Directory per Document for Output **************************/
            dir_per_term = new File(ANNOTATED_SENTENCES_CORPUS + "//" + document_name);
            dir_per_term.mkdir();
            /************* End Making Directory per Document for Output **************************/

            file_contents = fileManager.fileReader(input_dir + "//" + file_in_list).toString();

            while ((index = file_contents.indexOf("<Sentence>", index)) != -1)
            {
                int temp = index + 10;
                index = file_contents.indexOf("</Sentence>", temp);
                sentence_before_removing_lookup_tags = file_contents.substring(temp, index);

                if ((sentence_before_removing_lookup_tags.contains("<Lookup>")) &&
                        (sentence_before_removing_lookup_tags.length() < 1000) &&
                        !(sentence_before_removing_lookup_tags.contains("MathType"))) // MathType keyword represents the Equations in Text. Sentences with Equations are not processed.
                {
                    int index1 = 0;
                    int index2 = 0;
                    int index3 = 0;
                    String term = "";
                    Vector<String> terms_in_a_sentence = new Vector();

                    while ((index1 = sentence_before_removing_lookup_tags.indexOf("<Lookup>", index1)) != -1)
                    {
                            index1 = index1 + 8;
                            index2 = sentence_before_removing_lookup_tags.indexOf("</Lookup>", index1);
                            index3 = sentence_before_removing_lookup_tags.indexOf("<Lookup>", index1);
                            term = sentence_before_removing_lookup_tags.substring(index1, index2);

                            if (index3 < index2)
                            {
                                String output = strMan.pattern_replacer(term, "<Lookup>", "");
                                term = strMan.pattern_replacer(output, "</Lookup>", "");
                                term = term.trim();
                            }
                            //
                            // TODO: Check why if term contains "/", "\" or "}", the program stucks.
                            //
                            if ((!(term.contains("/"))) && (!(term.contains("{"))) &&(!(term.contains("\\"))) &&
                                    (!(term.contains("+"))) && (!(term.contains("*"))))
                            {
                                terms_in_a_sentence.add(term);
                            }
                    }

                    String output = strMan.pattern_replacer(sentence_before_removing_lookup_tags, "<Lookup>", "");
                    String sentence_after_removing_lookup_tags = strMan.pattern_replacer(output, "</Lookup>", "");
                    sentence_after_removing_lookup_tags = sentence_after_removing_lookup_tags.replaceAll("'", "");

                    if (sentence_after_removing_lookup_tags.length() > MAX_SENTENCE_LENGTH)
                    {
                        System.out.println("Sentence too long, Doc ID = " + document_id + "Sentence : " + sentence_after_removing_lookup_tags);
                        sentence_after_removing_lookup_tags = "SENTENCE TOO LONG";
                    }


                    for (int j = 0; j < terms_in_a_sentence.size(); j++)
                    {
                        String candidate_term = (String) terms_in_a_sentence.elementAt(j);
                        String sentence = sentence_after_removing_lookup_tags;
                        sentence = sentence.replaceAll(candidate_term, "MAIN_TERM");


                        for (int k = 0; k < terms_in_a_sentence.size(); k++)
                        {
                            sentence = sentence.replace(terms_in_a_sentence.elementAt(k).toString(),
                                    "Term_" + terms_in_a_document.size() + "_" +
                                    k);
                        }
                        terms_in_a_document.add(candidate_term);

                        String[] doc_table_1 = {"Doc_ID", "Local_Sen_ID", "Value"};
                        String[] doc_table_value_1 = {Integer.toString(document_id), Integer.toString(sentence_count), sentence_after_removing_lookup_tags};
                        sql_handler.insert_table("sentences_diseases_gad", doc_table_1, doc_table_value_1);

                        sentence_id = sql_handler.get_sentence_id_for_disease_gad(document_id, sentence_count);

                        String[] doc_table_2 = {"Disease", "Doc_ID", "Sen_ID"};
                        String[] doc_table_value_2 = {candidate_term, Integer.toString(document_id), Integer.toString(sentence_id)};
                        sql_handler.insert_table("diseases_gad", doc_table_2, doc_table_value_2);


                        if (sentence_count < 10)
                        {
                            fileManager.fileWriter(sentence,
                                    ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//00"
                                    + sentence_count + "__" + candidate_term + ".txt".trim());
                        }
                        else if (sentence_count < 100)
                        {
                            fileManager.fileWriter(sentence,
                                    ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//0"
                                    + sentence_count + "__" + candidate_term + ".txt".trim());
                        }
                        else
                        {
                            fileManager.fileWriter(sentence,
                                    ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//"
                                    + sentence_count + "__" + candidate_term + ".txt".trim());
                        }

                        sentence_count++;
                    }
                    terms_in_a_sentence.clear();
                }
            }
        }
        sql_handler.close_db();
    }


    public void countering_lookup_for_env_fac_gad() throws Exception
    {
        String[] tables =
        {
            "env_fac_gad", "sentences_env_fac_gad"
        };

        sql_handler.initialize_parameters(database_url, username, password);
        sql_handler.connect_db();

        BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
        String input;
        try
        {
            System.out.println(
                    "*****************************************************");
            System.out.println("You have opted to perform Sentence Splitting + Disease Name Extraction\n");
            System.out.print(
                    "WARNING: Your previous data in env_factors_gad and sentences_env_fac_gad \n " +
                    " tables will be erased \n" +
                    " Enter 'y' to continue or 'n' to exit : ");
            input = in.readLine();
            input = input.trim();
            if ((input.equals("Y")) || (input.equals("y")))
            {
                sql_handler.delete_all(tables);
                sql_handler.reset_auto_inc(tables);
            }
        }
        catch (Exception ex)
        {
            System.out.println("Exception while taking input from user : " + ex.getMessage());
        }

        String input_dir = ANNOTATED_DOCUMENTS_CORPUS;


        File file_dir = new File(input_dir);
        String[] list_files = file_dir.list();
        int no_of_files = list_files.length;

        if (no_of_files == 0)
        {
            System.out.println("Sorry! There are no input files to be processed");
            System.exit(0);
        }

        int document_id = 0;
        int sentence_id = 0;

        String file_contents = "";

        for (String file_in_list: list_files)
        {
            int sentence_count = 0;
            int index = 0;

            String sentence_before_removing_lookup_tags = "";
            Vector terms_in_a_document = new Vector();
            File dir_per_term;

            String document_name = file_in_list.substring(0, file_in_list.length() - 4);
            document_id = sql_handler.get_document_id_per_doc_value(document_name);


            /************* Making Directory per Document for Output **************************/
            dir_per_term = new File(ANNOTATED_SENTENCES_CORPUS + "//" + document_name);
            dir_per_term.mkdir();
            /************* End Making Directory per Document for Output **************************/

            file_contents = fileManager.fileReader(input_dir + "//" + file_in_list).toString();

            while ((index = file_contents.indexOf("<Sentence>", index)) != -1)
            {
                int temp = index + 10;
                index = file_contents.indexOf("</Sentence>", temp);
                sentence_before_removing_lookup_tags = file_contents.substring(temp, index);

                if ((sentence_before_removing_lookup_tags.contains("<Lookup>")) &&
                        (sentence_before_removing_lookup_tags.length() < 1000) &&
                        !(sentence_before_removing_lookup_tags.contains("MathType")))
                {
                    int index1 = 0;
                    int index2 = 0;
                    int index3 = 0;
                    String term = "";
                    Vector<String> terms_in_a_sentence = new Vector();

                    while ((index1 = sentence_before_removing_lookup_tags.indexOf("<Lookup>", index1)) != -1)
                    {
                            index1 = index1 + 8;
                            index2 = sentence_before_removing_lookup_tags.indexOf("</Lookup>", index1);
                            index3 = sentence_before_removing_lookup_tags.indexOf("<Lookup>", index1);
                            term = sentence_before_removing_lookup_tags.substring(index1, index2);

                            if (index3 < index2)
                            {
                                String output = strMan.pattern_replacer(term, "<Lookup>", "");
                                term = strMan.pattern_replacer(output, "</Lookup>", "");
                                term = term.trim();
                            }
                            //
                            // TODO: Check why if term contains "/", "\" or "}", the program stucks.
                            //
                            if ((!(term.contains("/"))) && (!(term.contains("{"))) &&(!(term.contains("\\"))) &&
                                    (!(term.contains("+"))) && (!(term.contains("*"))))
                            {
                                terms_in_a_sentence.add(term);
                            }
                    }

                    String output = strMan.pattern_replacer(sentence_before_removing_lookup_tags, "<Lookup>", "");
                    String sentence_after_removing_lookup_tags = strMan.pattern_replacer(output, "</Lookup>", "");
                    sentence_after_removing_lookup_tags = sentence_after_removing_lookup_tags.replaceAll("'", "");

                    if (sentence_after_removing_lookup_tags.length() > MAX_SENTENCE_LENGTH)
                    {
                        System.out.println("Sentence too long, Doc ID = " + document_id + "Sentence : " + sentence_after_removing_lookup_tags);
                        sentence_after_removing_lookup_tags = "SENTENCE TOO LONG";
                   }


                    for (int j = 0; j < terms_in_a_sentence.size(); j++)
                    {
                        String candidate_term = (String) terms_in_a_sentence.elementAt(j);
                        String sentence = sentence_after_removing_lookup_tags;
                        sentence = sentence.replaceAll(candidate_term, "MAIN_TERM");


                        for (int k = 0; k < terms_in_a_sentence.size(); k++)
                        {
                            sentence = sentence.replace(terms_in_a_sentence.elementAt(k).toString(),
                                    "Term_" + terms_in_a_document.size() + "_" +
                                    k);
                        }
                        terms_in_a_document.add(candidate_term);

                        String[] doc_table_1 = {"Doc_ID", "Local_Sen_ID", "Value"};
                        String[] doc_table_value_1 = {Integer.toString(document_id), Integer.toString(sentence_count), sentence_after_removing_lookup_tags};
                        sql_handler.insert_table("sentences_env_fac_gad", doc_table_1, doc_table_value_1);

                        sentence_id = sql_handler.get_sentence_id_for_env_fac_gad(document_id, sentence_count);

                        String[] doc_table_2 = {"Env_Fac_Gad_Name", "Doc_ID", "Sen_ID"};
                        String[] doc_table_value_2 = {candidate_term, Integer.toString(document_id), Integer.toString(sentence_id)};
                        sql_handler.insert_table("env_fac_gad", doc_table_2, doc_table_value_2);


                        if (sentence_count < 10)
                        {
                            fileManager.fileWriter(sentence,
                                    ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//00"
                                    + sentence_count + "__" + candidate_term + ".txt".trim());
                        }
                        else if (sentence_count < 100)
                        {
                            fileManager.fileWriter(sentence,
                                    ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//0"
                                    + sentence_count + "__" + candidate_term + ".txt".trim());
                        }
                        else
                        {
                            fileManager.fileWriter(sentence,
                                    ANNOTATED_SENTENCES_CORPUS + "//" + document_name + "//"
                                    + sentence_count + "__" + candidate_term + ".txt".trim());
                        }

                        sentence_count++;
                    }
                    terms_in_a_sentence.clear();
                }
            }
        }
        sql_handler.close_db();
    }

}